!! Copyright (C) 2009,2010,2011,2012  Marco Restelli
!!
!! This file is part of:
!!   FEMilaro -- Finite Element Method toolkit
!!
!! FEMilaro is free software; you can redistribute it and/or modify it
!! under the terms of the GNU General Public License as published by
!! the Free Software Foundation; either version 3 of the License, or
!! (at your option) any later version.
!!
!! FEMilaro is distributed in the hope that it will be useful, but
!! WITHOUT ANY WARRANTY; without even the implied warranty of
!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
!! General Public License for more details.
!!
!! You should have received a copy of the GNU General Public License
!! along with FEMilaro; If not, see <http://www.gnu.org/licenses/>.
!!
!! author: Marco Restelli                   <marco.restelli@gmail.com>

!>\brief
!! Interface to MPI.
!!
!! \n
!!
!! This is an interface to MPI functions and MPI related utilities.
!! Notice however that MPI specific variables (such as buffers,
!! communicators, tags and so on) must be defined where they are used.
!!
!! One of the main advantages of this module is that it allows an easy
!! switch between the two syntaxes
!! \code
!!   use mpi
!! \endcode
!! and
!! \code
!!   include "mpif.h"
!! \endcode
!! The first syntax should be preferred, but it requires MPI to be
!! compiled with the same compiler used for the application. The
!! second form does not provides the additional checks of fortran 90,
!! but it is the only choice when the mpi.mpd file is not available
!! for the chosen compiler.
!!
!! \bug \c mpi_type_create_f90_real has a bug in mpich2, so that one
!! has to use \c mpi_double_precision. The bug is fixed in version 1.3
!! (see https://trac.mcs.anl.gov/projects/mpich2/ticket/1028).
!<----------------------------------------------------------------------
module mod_mpi_utils

!-----------------------------------------------------------------------

 use mod_messages, only: &
   mod_messages_initialized, &
   error,   &
   warning, &
   info

 use mod_kinds, only: &
   mod_kinds_initialized, &
   wp_p, wp_r

!-----------------------------------------------------------------------

 ! Select here the desired bindings: f77 or f90

 !use mpi
 implicit none
 include "mpif.h"
 external :: mpi_init, mpi_initialized

!-----------------------------------------------------------------------

! Module interface

 public :: &
   mod_mpi_utils_constructor, &
   mod_mpi_utils_destructor,  &
   mod_mpi_utils_initialized, &
   mpi_logical, mpi_integer, wp_mpi, &
   mpi_comm_world, mpi_status_size,  &
   mpi_undefined, mpi_comm_split,    &
   mpi_comm_free,                    &
   mpi_sum, mpi_max, mpi_lor,        &
   mpi_init, mpi_finalize,           &
   mpi_thread_single, mpi_thread_multiple, &
   mpi_comm_size, mpi_comm_rank,     &
   mpi_barrier,                      &
   mpi_bcast,                        &
   mpi_isend, mpi_irecv,             &
   mpi_send,  mpi_recv,              &
   mpi_request_null,                 &
   mpi_wait, mpi_waitall,            &
   mpi_sendrecv,                     &
   mpi_gather, mpi_allgather,        &
   mpi_reduce, mpi_allreduce,        &
   mpi_alltoall, mpi_alltoallv!,      &
   !mpix_iallreduce

 private

!-----------------------------------------------------------------------

 external :: mpi_finalize, mpi_comm_size, mpi_comm_rank, mpi_barrier,  &
   mpi_isend, mpi_irecv, mpi_send, mpi_recv, mpi_alltoall,             &
   mpi_alltoallv, mpi_bcast, mpi_reduce, mpi_allreduce, mpi_sendrecv,  &
   mpi_wait, mpi_waitall, mpi_gather, mpi_allgather, mpi_comm_split,   &
   mpi_comm_free, &
   !mpix_iallreduce,                                                    &
  ! private functions
   mpi_type_create_f90_real

! Module variables
 ! public members
 integer, protected :: wp_mpi !< MPI representation of wp
 logical, protected ::               &
   mod_mpi_utils_initialized = .false.
 ! private members
 character(len=*), parameter :: &
   this_mod_name = 'mod_mpi_utils'

!-----------------------------------------------------------------------

contains

!-----------------------------------------------------------------------

 subroutine mod_mpi_utils_constructor()

  logical :: already_initialized_mpi
  integer :: ierr
  character(len=100) message
  character(len=*), parameter :: &
    this_sub_name = 'constructor'

   !Consistency checks ---------------------------
   if( (mod_messages_initialized.eqv..false.) .or. &
          (mod_kinds_initialized.eqv..false.) ) then
     call error(this_sub_name,this_mod_name, &
                'Not all the required modules are initialized.')
   endif
   if(mod_mpi_utils_initialized.eqv..true.) then
     call warning(this_sub_name,this_mod_name, &
                  'Module is already initialized.')
   endif
   call mpi_initialized(already_initialized_mpi,ierr)
   if(.not.already_initialized_mpi)          &
     call error(this_sub_name,this_mod_name, &
       'MPI must be initialized before this module.')
   !----------------------------------------------

   ! Compiler bug: mpich2 has problems with mpi_type_create_f90_real, ! this
   ! should be fixed from version 1.3. Also OpenMPI seems to have problems with
   ! mpi_type_create_f90_real for quadruple precision. So, one can use the
   ! mpi_* predefined constants whenever mpi_type_create_f90_real does not work.
   wp_mpi = mpi_double_precision
   !wp_mpi = mpi_real16
   !call mpi_type_create_f90_real(wp_p,wp_r,wp_mpi,ierr)

   write(message,*) 'mpi representation of wp = ', wp_mpi
   call info(this_sub_name,this_mod_name,message)

   mod_mpi_utils_initialized = .true.
 end subroutine mod_mpi_utils_constructor

!-----------------------------------------------------------------------
 
 subroutine mod_mpi_utils_destructor()
  character(len=*), parameter :: &
    this_sub_name = 'destructor'
   
   !Consistency checks ---------------------------
   if(mod_mpi_utils_initialized.eqv..false.) then
     call error(this_sub_name,this_mod_name, &
                'This module is not initialized.')
   endif
   !----------------------------------------------

   mod_mpi_utils_initialized = .false.
 end subroutine mod_mpi_utils_destructor

!-----------------------------------------------------------------------

end module mod_mpi_utils

